package project3cs424.src;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.tidy.Tidy;


public class UFOScraper {
	
	public static void main(String[] args)throws IOException
	{
		
		ArrayList<String> exclusions=new ArrayList<String>();
		List<String> links=Collections.synchronizedList(new ArrayList<String>());
		UFOScraper s=new UFOScraper();
		String inputUrl=args[0];
		String baseUrl="http://www.nuforc.org/webreports";
		links.add(" ###"+inputUrl);
		Iterator<String> linksIterator=links.iterator();
		
		exclusions.add("ALBERTA.CANADA");
		exclusions.add("BRITISH COLUMBIA.CAN");
		exclusions.add("MANITOBA.CANADA");
		exclusions.add("NEW BRUNSWICK.CAN");
		exclusions.add("NEWFOUNDLAND.CAN");
		exclusions.add("NOVA SCOTIA.CAN");
		exclusions.add("NORTHWEST TERRITORY");
		exclusions.add("ONTARIO.CAN");
		exclusions.add("PRINCE EDW ISLAND");
		exclusions.add("PROV OF QUE.CAN");
		exclusions.add("QC");
		exclusions.add("SASKATCHEWAN.CAN");
		exclusions.add("SK");
		exclusions.add("UNSPECIFIED.INTERNATIONAL");
		exclusions.add("YUKON");
		exclusions.add("YT");
		exclusions.add("VIRGIN ISLANDS");
		exclusions.add("entries");
		
		for(int j=0;j<links.size();j++)
		{
		
			String inputText=links.get(j).toString();
			inputUrl=inputText.split("###")[1].trim();
			String outputFileName=inputText.split("###")[0].trim();
			
			URL url=new URL(inputUrl);

			System.out.println(outputFileName);
			if(outputFileName.equals(""))
			{
				outputFileName="test";
			}
			else if(outputFileName.contains("/"))
			{
				outputFileName=outputFileName.replaceAll("/", ".");
			}
			else if(outputFileName.contains(","))
			{
				outputFileName=outputFileName.replaceAll(",", ".");
			}
			System.out.println(outputFileName);
			if(exclusions.contains(outputFileName))
			{
				continue;
			}
			BufferedWriter outputWriter=new BufferedWriter(new FileWriter(new File("output/"+outputFileName),true));
			InputStreamReader isr=new InputStreamReader(url.openStream());
			Tidy tidy=new Tidy();
			tidy.setQuiet(true);
			tidy.setShowWarnings(false);
			tidy.setForceOutput(true);
			
			Document document=tidy.parseDOM(isr, null);
			
			NodeList tableNodes=document.getElementsByTagName("table");
//			for(int i=0;i<tableNodes.getLength();i++)
//			{
//				System.out.println(tableNodes.item(i).getNodeName());
//			}
			Node tableNode=tableNodes.item(0);
			
			NodeList tableNodeChildren=tableNode.getChildNodes();
			Node tBodyNode=null;
			
			for(int i=0;i<tableNodeChildren.getLength();i++)
			{
				Node node=tableNodeChildren.item(i);
				if(node.getNodeName().equals("tbody"))
				{
					tBodyNode=node;
				}
			}
			
			Node trNode=null;
			NodeList tBodyChildren=tBodyNode.getChildNodes();
			
			for(int i=0;i<tBodyChildren.getLength();i++)
			{
				Node node=tBodyChildren.item(i);
				if(node.getNodeName().equals("tr"))
				{
					trNode=node;
					
					NodeList trNodeList=trNode.getChildNodes();
					trNodeList.getLength();
					if(trNode.getChildNodes().getLength()==2)
					{
						//process main page
						Node tdTag=trNode.getFirstChild();
						Node fontTag=tdTag.getFirstChild();
						Node anchorTag=fontTag.getFirstChild();
					//	System.out.println(anchorTag.getNodeName());
						String fileName=anchorTag.getChildNodes().item(0).getNodeValue();
						System.out.println(fileName);
						NamedNodeMap properties=anchorTag.getAttributes();
						Node hrefAttribute=properties.getNamedItem("href");
				//		System.out.println(hrefAttribute.getNodeValue());
						links.add(fileName+"###"+baseUrl+"/"+hrefAttribute.getNodeValue());
						
						//s.processUrl(baseUrl+hrefAttribute.getNodeValue());
						
					}
					else
					{
						
						//extract data
						NodeList trNodeChildren=trNode.getChildNodes();
						
						String summary="";
						if(trNodeChildren.getLength()==7)
						{
							
							for(int k=0;k<trNodeChildren.getLength();k++)
							{
								String textToExtract;
								Node tdNode=trNodeChildren.item(k);
								if(tdNode.hasChildNodes())
								{
								
									Node fontNode=tdNode.getFirstChild();
									Node textNode;
									if(k==0)
									{
										//process link here
										Node anchorNode=fontNode.getFirstChild();
										if(anchorNode.getNodeName().equals("a"))
										{
											NamedNodeMap anchorAttributes=anchorNode.getAttributes();
											Node hrefAttribute=anchorAttributes.getNamedItem("href");
											summary=getSummary(baseUrl+"/"+hrefAttribute.getNodeValue());
											
											if(anchorNode.hasChildNodes())
											{
												textNode=anchorNode.getFirstChild();
										//		System.out.println(textNode.getNodeName());
												textToExtract=textNode.getNodeValue().trim();
												if(textToExtract.contains(" "))
												{
													String textParts[]=textToExtract.split(" ");
													if(textParts[0].contains("/"))
													{
														String dateParts[]=textParts[0].split("/");
														if(dateParts.length==3)
														{
															int year=Integer.parseInt(dateParts[2]);
															if(!(year>=0 && year<=10))
															{
																continue;
															}
														}
														
													}
														
												}
												else
												{
													if(textToExtract.contains("/"))
													{
														String dateParts[]=textToExtract.split("/");
														if(dateParts.length==3)
														{
															int year=Integer.parseInt(dateParts[2]);
															if(!(year>=0 && year<=10))
															{
																continue;
															}
														}
													}
												}
												
												
												
											}
											else
											{
												textToExtract=" ";
											}
											
										}
										else
										{
											textToExtract=anchorNode.getNodeValue();
										}
										
									}
									else if(k==5)
									{
										if(summary.equals(""))
										{
											if(fontNode.hasChildNodes())
											{
												textNode=fontNode.getFirstChild();
												textToExtract=textNode.getNodeValue();	
											}
											else
											{
												textToExtract="";
											}
												
										}
										else
										{
											textToExtract=summary;	
										}
										
									}
									else
									{
										if(fontNode.hasChildNodes())
										{
											textNode=fontNode.getFirstChild();
											textToExtract=textNode.getNodeValue();	
										}
										else
										{
											textToExtract="";
										}
										
									}
									
																	
								}
								else
								{
									textToExtract="";
								}
								if(k==trNodeChildren.getLength()-1)
								{
									outputWriter.write("\t"+textToExtract+"\n");
									//System.out.println(textToExtract);
								}
								else if(k==0)
								{
								outputWriter.write(textToExtract);
								}
								else
								{
									
									outputWriter.write("\t"+textToExtract);
								}
								
							}
								
							
							
//							String dateTime=trNodeChildren.item(0)..getTextContent();
//							String city=trNodeChildren.item(1).getTextContent();
//							String state=trNodeChildren.item(2).getTextContent();
//							String shape=trNodeChildren.item(3).getTextContent();
//							String duration=trNodeChildren.item(4).getTextContent();
//							String summary=trNodeChildren.item(5).getTextContent();
//							String posted=trNodeChildren.item(6).getTextContent();
//							outputWriter.write(dateTime+"###"+city+"###"+state+"###"+shape+"###"+duration+"###"+summary+"###"+posted+"\n");
						}
									
					}
					
				}
				
			}
			outputWriter.close();

			File fileToDelete=new File("test");
			if(fileToDelete.delete())
			{
				System.out.println("test deleted");
			}
				
							
		}
		
				
	}
	
	private static String getSummary(String string) {
		
		String summaryText="";
		try {
			URL url=new URL(string);
			InputStreamReader isr=new InputStreamReader(url.openStream());
			Tidy tidy=new Tidy();
			tidy.setQuiet(true);
			tidy.setShowWarnings(false);
			tidy.setForceOutput(true);
			
			Document document=tidy.parseDOM(isr, null);
			
			NodeList tbodyNodes=document.getElementsByTagName("tbody");
			if(tbodyNodes.getLength()!=0)
			{
				Node tbodyNode=tbodyNodes.item(0);
				NodeList tbodyChildren=tbodyNode.getChildNodes();
				Node trNode=tbodyChildren.item(1);
				NodeList trNodeChildren=trNode.getChildNodes();
				Node tdNode=trNodeChildren.item(0);
				
				if(tdNode.hasChildNodes())
				{
				//	System.out.println(tdNode.getNodeName());
					Node fontNode=tdNode.getChildNodes().item(0);
					summaryText=getAllTextFromNode(fontNode);
				}
				else
				{
			//		System.out.println(tdNode.getNodeName());
					summaryText=getAllTextFromNode(tdNode);
				}
			//	System.out.println(summaryText);
	
			}
								
			} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return summaryText;
	}

	void processUrl(String urlInPage)
	{
		
	}
	
	static String getAllTextFromNode(Node node)
	{
		String summary="";
		if(node.hasChildNodes())
		{
			NodeList nodeChildren=node.getChildNodes();
			for(int i=0;i<nodeChildren.getLength();i++)
			{
				Node child=nodeChildren.item(i);
				summary+=" "+getAllTextFromNode(child);
			}
			
		}
		else if(node.getNodeName().equals("#text"))
		{
			summary+=" "+node.getNodeValue();
		}
		return summary;
	}

}
